/*
 * Copyright (c) 2018-2023 北京华恒盛世科技有限公司
 * QUICKPOOL is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *     http://license.coscl.org.cn/MulanPSL2
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
*/

#include "subcmd.h"

#define QPAI_FILE "qp.ai"
#define MAX_SUBMIT_OPTIONS  200

static char *sstrCommand;
static bool_t sboolEmptyCmd = TRUE;

static int fill_submit_request(int, char **, enum submitType, struct request *);
static bool_t set_job_command(char **);
static int parse_submit_file(FILE *, unsigned int *, char ***);
static void job_sub_usage(enum submitType);

/**
 * @brief       qp job sub handler, to submit a job to the cluster.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 *
 * @retval      0             #1: succeed
 * @retval      1             #2: input option is wrong or kind of input argument not specified
 * @retval      2             #3: argument has syntax error
 * @retval      3             #4: failed to modify the job
 * @retval      4             #5: not enough memory
 */
int
cmd_submit_job(int argc, char **argv)
{
    struct request req;

    enum submitType submit;
    if (0 == strcmp(argv[0], "sub")) {
        submit = CMD_SUBMIT_JOB;
    } else if (0 == strcmp(argv[0], "restart")) {
        submit = CMD_RESTART_JOB;
    } else {
        fprintf(stderr,  "Unknown subcmd <%s> specified for submitting job.\n", argv[0]);
        exit(1);
    }

    if (set_submit_request(argc, argv, submit, &req) < 0 ) {
        fprintf(stderr,  ". Job not submitted.\n");
        exit(2);
    }

    time_t beginTime = 0;
    time_t termTime = 0;

    unsigned int i;
    for (i=0; i<req.number; i++) {
        switch (req.keyvalues[i].key) {
        case REQUEST_KEY_SUBMIT_BEGINTIME:
            beginTime = atol(req.keyvalues[i].value);
            break;
        case REQUEST_KEY_SUBMIT_TERMTIME:
            termTime = atol(req.keyvalues[i].value);
            break;
        }

        if (beginTime > 0 && termTime > 0) {
            break;
        }
    }

    struct QuickPoolSubmitReply reply;
    memset(&reply, 0, sizeof(struct QuickPoolSubmitReply));

    JOBID_T jobId = -1;
    jobId = client_submit_job(&req, &reply, NULL);
    if (jobId < 0) {

        if (0 < strlen(reply.error_message)) {
            fprintf(stderr, "%s Job not submitted.\n", reply.error_message);
        } else {
            fprintf(stderr, "Job not submitted due to above error.\n");
        }
        gf_xdr_free(libxdr_submit_reply, &reply);

        exit(3);
    }

    if (NULL != getenv("SUBMIT_QUIET")) { // silent submit, diplay nothing after submission.

        gf_xdr_free(libxdr_submit_reply, &reply);
        exit(0);
    }

    char timebuf[TIMESTRINGLEN];
    char strJobId[32];
    if (NULL != getenv("SUBMIT_STDERR")) { // print out message to stderr.

        fprintf(stderr, "Job <%s> is submitted to queue <%s>.\n",
                jobid_2string(jobId, strJobId), reply.queue);

        if (0 < beginTime) {
            fprintf(stderr, "Job will be scheduled after %s\n",
                    ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &beginTime, timebuf));
        }

        if (0 < termTime) {
            fprintf(stderr, "Job will be terminated by %s\n",
                    ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &termTime, timebuf));
        }

        fflush(stderr);
    } else { // print out message to stdout.

        fprintf(stdout, "Job <%s> is submitted to queue <%s>.\n",
                jobid_2string(jobId, strJobId), reply.queue);

        if (0 < beginTime) {
            fprintf(stdout, "Job will be scheduled after %s\n",
                    ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &beginTime, timebuf));
        }

        if (0 < termTime) {
            fprintf(stdout, "Job will be terminated by %s\n",
                    ctime_2string(CTIME_FORMAT_a_b_d_T_Y, &termTime, timebuf));
        }

        fflush(stdout);
    }
    gf_xdr_free(libxdr_submit_reply, &reply);

    exit(0);
} // end function cmd_submit_job

/**
 * @brief       parse all input arguments and set job submit request.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 * @param[in]   submit        #3: submit, modify or restart a job
 * @param[out]  req           #4: request to be set
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: input option is wrong or kind of input argument not specified
 */
int
set_submit_request(int argc, char **argv, enum submitType submit, struct request *req)
{
    memset(req, 0, sizeof(struct request));
    req->keyvalues = (struct keyvalue *)calloc(MAX_SUBMIT_OPTIONS, sizeof(struct keyvalue));
    if (NULL == req->keyvalues) {
        fprintf(stderr, "calloc(%ld) failed due to %m.\n",
                MAX_SUBMIT_OPTIONS*sizeof(struct keyvalue));
        return -1;
    }

    if (fill_submit_request(argc, argv, submit, req) == -1) {
        return -1;
    }

    unsigned int n = req->number;

    if (CMD_MODIFY_JOB == submit) { // modify a job

        if (argc == optind + 1) {
            req->keyvalues[n].key = REQUEST_KEY_MODIFY_JOBID;
            req->keyvalues[n].value = copy_string(argv[optind]);
            n++;
        } else { // not specify job id
            job_sub_usage(submit);
            return -1;
        }
    } else if (CMD_RESTART_JOB == submit) { // restart a job
        if (argc == optind + 1) {
            req->keyvalues[n].key = REQUEST_KEY_RESTART_CHKPNTDIR;
            req->keyvalues[n].value = copy_string(argv[optind]);
            n++;
        } else if (argc == optind + 2) {

            JOBID_T arrayJobId;
            if (get_one_jobid(argv[optind+1], &arrayJobId))
                return -1;

            char strJobId[32];
            static char chkDir[PATH_MAX+1];
            snprintf(chkDir, PATH_MAX, "%s/%s",
                     argv[optind], jobid_2longstring(arrayJobId, strJobId));

            req->keyvalues[n].key = REQUEST_KEY_RESTART_CHKPNTDIR;
            req->keyvalues[n].value = copy_string(chkDir);
            n++;
        } else { // not specify the cknpnt dir
            job_sub_usage(submit);
            return -1;
        }
    } else { // submit a job
        unsigned int i;
        char *jobName=NULL;
        unsigned int num;

        if (argc >= optind + 1) {

            sboolEmptyCmd = FALSE;
            if (!set_job_command(argv+optind)) {
                return -1;
            }
        } else { // not specify command, parse a job script file or get input from stdin 

            char **embedArgv;
            unsigned int embedArgc = 0;
            if (parse_submit_file(stdin, &embedArgc, &embedArgv) < 0) {
                return -1;
            }

            optind = 1;
            if (1 < embedArgc) {

                req->number = n;
                if (fill_submit_request(embedArgc, embedArgv, submit, req) == -1) {
                    return -1;
                }
                n = req->number;
            }
        }

        strip_spaces(sstrCommand, FALSE);
        req->keyvalues[n].key = REQUEST_KEY_SUBMIT_COMMAND;
        req->keyvalues[n].value = copy_string(sstrCommand);
        n++;

        FREEUP(sstrCommand);

        if (sboolEmptyCmd) {

            fprintf(stderr, "No command is specified");
            return -1;
        }       

        bool_t memLimFlag = FALSE, runLimFlag = FALSE;
        for (i = 0; i < n; i++) {
            if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_JOBNAME) {
                jobName = req->keyvalues[i].value;
            }
            if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_RSSLIMIT) {
                memLimFlag = TRUE;
            }  
            if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_RUNLIMIT) {
                runLimFlag = TRUE;
            }  
            if (NULL != jobName && memLimFlag && runLimFlag) {
                break;
            }
        }

        if ((NULL == jobName) ||  (runLimFlag == TRUE && memLimFlag == TRUE)) {
            req->number = n;
            return 0;
        }

        char qpaiFilename[PATH_MAX+1];
        snprintf(qpaiFilename, PATH_MAX, "%s/%s",
                 baseParams[BASE_WORKDIR].string_value, QPAI_FILE);
        if (access(qpaiFilename, R_OK) != 0) { // no ai file, nothing more need to do.
            req->number = n;
            return 0;
        }        
        
        FILE *aiFp;
        aiFp = fopen(qpaiFilename, "r");
        if (aiFp == NULL) {
            fprintf(stderr, "Automatic parameter configuration is ignored this time because opening %s failed!", QPAI_FILE);
            req->number = n;
            return 0;                
        }

        // read the ai file to update job's memory requiremnt
        struct record *logRec;
        while ((logRec = read_file_line(aiFp, &num)) != NULL) {

            bool_t foundJobName = FALSE; 
            for (i = 0; i < logRec->numKVs; i++) {
                if (logRec->KVs[i].key ==  RECORD_KEY_JOB_NAME) {
                    if (NULL != jobName && 0 == strcmp(jobName, logRec->KVs[i].value)) {
                        foundJobName = TRUE;
                    }
                    break;
                }                 
            }

            if (!foundJobName) {  
                free_key_value(logRec->numKVs, &(logRec->KVs));
                FREEUP(logRec->KVs); 
                logRec->numKVs = 0;
                continue;
            }

            for (i = 0; i < logRec->numKVs; i++) {
                if ((logRec->KVs[i].key ==  RECORD_KEY_JOB_RUNTIME) && (runLimFlag == FALSE)) {
                    req->keyvalues[n].key = REQUEST_KEY_SUBMIT_RUNLIMIT;
                    req->keyvalues[n].value = copy_string(logRec->KVs[i].value);
                    n++;
                }   
                if ((logRec->KVs[i].key ==  RECORD_KEY_USAGE_MEM) && (memLimFlag == FALSE)) {
                    int memlimit;
                    memlimit =gf_atoi(logRec->KVs[i].value, INFINIT_INT/1024, 1);
                    if (INFINIT_INT != memlimit) {
                        char buf[20];
                        memlimit *= 1024;
                        snprintf(buf, 20, "%d", memlimit);
                        req->keyvalues[n].key = REQUEST_KEY_SUBMIT_RSSLIMIT;
                        req->keyvalues[n].value = copy_string(buf);
                        n++;
                     }
                }  
            }  

            free_key_value(logRec->numKVs, &(logRec->KVs));
            FREEUP(logRec->KVs); 
            logRec->numKVs = 0;
        }

        fclose(aiFp);
    }
    req->number = n;

    return 0;
} // end function set_submit_request

/* ------------------------------------------------
 *
 *  below are static functions used in this file
 *
 * ------------------------------------------------ */

/**
 * @brief       parse job submit argument and set job command(sstrCommand).
 *
 * @param[in]   from          #1: start of argv for command
 *
 * @retval      TRUE          #1: succeed
 * @retval      FALSE         #2: not enough memory
 */
static bool_t
set_job_command(char **from)
{
    int i, size;
    for (i = 0, size = 0; from[i]; i++) {
        size += strlen(from[i]) + 1 + 4;
    }

    size += 1 + 1;

    sstrCommand = (char *)calloc(size, sizeof(char));
    if (NULL == sstrCommand) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", size);
        return FALSE;
    }

    if (NULL == commandParams[API_QUOTE_CMD].string_value) {
        strcpy(sstrCommand, from[0]);
        i = 1;
    } else {
        if ((strcasecmp(commandParams[API_QUOTE_CMD].string_value, "yes") == 0)
            || ((strcasecmp(commandParams[API_QUOTE_CMD].string_value, "y") == 0))) {
            memset(sstrCommand, '\0', size);
            i = 0;
        } else {
            strcpy(sstrCommand, from[0]);
            i = 1;
        }
    }

    for(; from[i] != NULL ;i++) {
        strcat(sstrCommand, " ");

        if (strchr(from[i], ' ')) {
            if (strchr(from[i], '"')) {
                strcat(sstrCommand, "'");
                strcat(sstrCommand, from[i]);
                strcat(sstrCommand, "'");
            } else {
                if (strchr(from[i], '$')) {
                    strcat(sstrCommand, "'");
                    strcat(sstrCommand, from[i]);
                    strcat(sstrCommand, "'");
                } else {
                    strcat(sstrCommand, "\"");
                    strcat(sstrCommand, from[i]);
                    strcat(sstrCommand, "\"");
                }
            }

            continue;
        }

        char *arg, *keepArg;
        arg = copy_string(from[i]);
        keepArg = arg;
        while (*arg) {
            char *endArg, *endArg1;

            endArg = strchr(arg, '"');
            endArg1 = strchr(arg, '\'');
            if (!endArg || (endArg1 && endArg > endArg1)) {
                endArg = endArg1;
            }

            endArg1 = strchr(arg, '\\');
            if ('$' == from[i][0]) {
                strcat(sstrCommand, "'");
            }
            if (!endArg || (endArg1 && endArg > endArg1)) {
                endArg = endArg1;
            }

            endArg1 = strchr(arg, '(');
            if (!endArg || (endArg1 && endArg > endArg1)) {
                endArg = endArg1;
            }

            endArg1 = strchr(arg, ')');
            if (!endArg || (endArg1 && endArg > endArg1)) {
                endArg = endArg1;
            }

            char endChar='\0';
            if (endArg) {
                endChar = *endArg;
                *endArg = '\0';
            }

            strcat(sstrCommand, arg);
            if ('$' == from[i][0]) {
                strcat(sstrCommand, "'");
            }

            if (endArg) {

                arg += strlen(arg) + 1;
                if ('\\' == endChar) {
                    strcat(sstrCommand, "\\\\");
                } else if ('"' == endChar) {
                    strcat(sstrCommand, "\\\"");
                } else if ('\'' == endChar) {
                    strcat(sstrCommand, "\\\'");
                } else if ('(' == endChar) {
                    strcat(sstrCommand, "\\\(");
                } else if (')' == endChar) {
                    strcat(sstrCommand, "\\)");
                }
            } else {
                arg += strlen(arg);
            }
        }

        FREEUP(keepArg);
    }

    return TRUE;
} // end function set_job_command

/**
 * @brief       parse one line and set submit option and arguement.
 *
 * @param[in]   line          #1: one line of file content
 * @param[out]  embedargc     #2: number of argument we got from the file
 * @param[out]  embedargv     #3: arguments we got from the file
 *
 * @retval      0             #1: succeed
 * @retval      1             #2: this line is job command
 * @retval      -1            #2: not enough memory
 */
static int
parse_submit_line(char *line, unsigned int *embedargc, char ***embedargv)
{
#define INCREASE 40

    static char **argBuf = NULL, *key;
    static unsigned int bufSize = 0;

    if (NULL == argBuf) {

        argBuf = (char **)calloc(INCREASE, sizeof(char *));
        if (NULL == argBuf) {
            fprintf(stderr, "calloc(%ld) failed due to %m.\n", INCREASE*sizeof(char *));
            return -1;
        }

        bufSize = INCREASE;
        *embedargc = 1;
        *embedargv = argBuf;

        argBuf[0] = (char *)"sub";
        key = (char *)"SUBMIT";

        argBuf[1] = NULL;
    }

    SKIPSPACE(line);
    if (*line == '\0') {
        return 0;
    }

    if (*line != '#') {
        sboolEmptyCmd = FALSE;
        return 1; // command line
    }

    ++line;
    if (*line == '!') {
        return 0;
    }

    SKIPSPACE(line);
    if (strncmp(line, key, strlen(key)) != 0) {
        return 0;
    }

    line += strlen(key);
    SKIPSPACE(line);
    if (*line != '-') {
        return 0;
    }

    while (TRUE) {

        char *sQuote, *dQuote, quoteMark;
        quoteMark = '"';
        if ((sQuote = strchr(line, '\'')) != NULL) {
            if ((dQuote = strchr(line, '"')) == NULL || sQuote < dQuote)
                quoteMark = '\'';
        }

        char *sp;
        if ((sp = get_string_inside(&line, quoteMark, quoteMark)) == NULL) {
            return 0;
        }

        if ('#' == *sp) { // a comment line, ignore it.
            FREEUP(sp);
            return 0;
        }

        if (*embedargc + 2 > bufSize) {
            bufSize += INCREASE;
            if ((argBuf = (char **) realloc(argBuf, bufSize * sizeof(char *))) == NULL) {
                fprintf(stderr, STR_FUNC_FAIL,__func__,"realloc" );
                return -1;
            }
            *embedargv = argBuf;
        }

        if ('\n' == sp[strlen(sp)-1]) {
            sp[strlen(sp)-1] = '\0';
        }

        argBuf[*embedargc] = sp;
        (*embedargc) ++;
        argBuf[*embedargc] = NULL;
    }

    return 0;
} // end function parse_submit_line

/**
 * @brief       read file and pass job submit option and arguement.
 *
 * @param[in]   from          #1: file to read or stdin
 * @param[out]  embedargc     #2: number of argument we got from the file
 * @param[out]  embedargv     #3: arguments we got from the file
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: not enough memory
 */
static int
parse_submit_file(FILE *from, unsigned int *embedargc, char ***embedargv)
{
    int ttyin=0;
    ttyin = isatty(fileno(from));
    if (ttyin) {
        fprintf(stdout, "submit>");
        fflush(stdout);
    }

    char line[PATH_MAX*10];
    int lineLen;
    char *sp;
    bool_t isSubmit=FALSE;

    sstrCommand = (char *)calloc(PATH_MAX*10+1, sizeof(char));
    if (NULL == sstrCommand) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", PATH_MAX*10);
        return FALSE;
    }

    sp = line;
    lineLen = 0;
    while (fgets(sp, 10 *PATH_MAX - lineLen -1, from) != NULL) {

        lineLen = strlen(line);
        if (0 == lineLen) {
            sp = line;
            continue;
        }

        if (line[0] == '#') {
            if (strstr(line, "SUBMIT") != NULL) {
                isSubmit = TRUE;
            }
        }

        if (isSubmit) {

            isSubmit = FALSE;
            if (line[lineLen-2] == '\\' && line[lineLen-1] == '\n') {
                lineLen -= 2;
                sp = line + lineLen;
                continue;
            }
        }

        int ret;
        ret = parse_submit_line(line, embedargc, embedargv);
        if (0 > ret) {
            return -1;
        }

        if (ttyin) {
            fprintf(stdout, "submit>");
            fflush(stdout);
        }

        if (1 == ret) {

            snprintf(sstrCommand+strlen(sstrCommand), PATH_MAX*10, "%s", line);
        }

        sp = line;
        lineLen = 0;
    }

    return 0;
} // end function parse_submit_file

/**
 * @brief       parse file transfer requirement and fill the submit request.
 *
 * @param[in]   arg           #1: file transfer requirement string
 * @param[out]  req           #2: request to be filled
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: input option is wrong or kind of input argument not specified
 */
static int
parse_execution_file(char *arg, struct request *req)
{
    char *p;
    char saveArg[PATH_MAX+1];
    char op[3], lf[PATH_MAX+1], rf[PATH_MAX+1];

    strncpy(saveArg, arg, PATH_MAX);
    if ((p = strstr(saveArg, "<<"))) {
        strncpy(op, "<<", 3);
    } else if ((p = strstr(saveArg, "><"))) {
        strncpy(op, "><", 3);
    } else if ((p = strstr(saveArg, "<>"))) {
        strncpy(op, "<>", 3);
    } else if ((p = strstr(saveArg, ">>"))) {
        strncpy(op, ">>", 3);
    } else if ((p = strstr(saveArg, "<"))) {
        strncpy(op, "<", 3);
    } else if ((p = strstr(saveArg, ">"))) {
        strncpy(op, ">", 3);
    } else {
        if (0 == strlen(op)) {
            fprintf(stderr, "There is no file operation specification in -f \"%s\". The valid operations are '<', '>', '<<', '>>', '<>', '><'",
                    saveArg);
        } else {
            fprintf(stderr, "Invalid file operation \"%s\" specification in -f \"%s\". The valid operations are '<', '>', '<<', '>>', '<>', '><'",
                    op, saveArg);
        }
        return -1;
    }

    memset(lf, 0, PATH_MAX+1);
    memset(rf, 0, PATH_MAX+1);
    memcpy(lf, saveArg, p - saveArg);
    memcpy(rf, p + strlen(op), strlen(saveArg) - strlen(lf) - strlen(op));

    if (strstr(lf, ">") || strstr(rf, ">") || strstr(rf, "<")) {
        fprintf(stderr, "Invalid file operation \"%s\" specification in -f \"%s\"", op, saveArg);
        return -1;
    }

    if (strlen(lf) == 0) {
        fprintf(stderr, "Invalid local file specification in -f \"%s\"", saveArg);
        return -1;
    }

    if ((lf[strlen(lf) - 1]) != ' ') {
        fprintf(stderr, "Invalid file operation \"%s\" specification in -f \"%s\"", op, saveArg);
        return -1;
    }

    strip_spaces(lf, FALSE);

    if (0 == strlen(lf)) {
        fprintf(stderr, "Invalid local file specification in -f \"%s\"", saveArg);
        return -1;
    }

    if (0 < strlen(rf)) {

        if (' ' != rf[0]) {
            fprintf(stderr, "Invalid file operation \"%s\" specification in -f \"%s\"", op, saveArg);
            return -1;
        }

        strip_spaces(rf, FALSE);

        if (strlen(rf) == 0) {
            strncpy(rf, lf, PATH_MAX);
        }
    } else {
        strncpy(rf, lf, PATH_MAX);
    }

    unsigned int n = req->number;
    unsigned int i;
    for (i=0; i<req->number; i++) {
        if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_NUM_XFILE) {
            break;
        }
    }

    unsigned int numFiles = 0;
    if (i < req->number) {
        numFiles = atoi(req->keyvalues[i].value);
        numFiles ++;
        snprintf(req->keyvalues[i].value, 12, "%d", numFiles);
    } else {
        numFiles ++;

        req->keyvalues[n].key = REQUEST_KEY_SUBMIT_NUM_XFILE;
        req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[n].value, 12, "%d", numFiles);
        n ++;
    }

    unsigned int size;
    size = strlen(lf) + strlen(rf) + strlen(op) + 3;
    req->keyvalues[n].key = REQUEST_KEY_SUBMIT_XFILE;
    req->keyvalues[n].value = (char *)calloc(size+1, sizeof(char));
    if (NULL == req->keyvalues[n].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", size+1);
        return -1;
    }
    snprintf(req->keyvalues[n].value, size, "%s %s %s", lf, op, rf);
    n ++;

    req->number = n;

    return 0;
} // end function parse_execution_file

/**
 * @brief       parse resource limit and fill the submit request.
 *
 * @param[in]   opt           #1: option, one of -c, -C, -D, -F, -M, -p, -S, -v, -W
 * @param[in]   submit        #2: submit, modify or restart a job
 * @param[in]   key           #3: resource limit key
 * @param[in]   arg           #4: resource limit requirement string
 * @param[out]  req           #5: request to be filled
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: input option is wrong or kind of input argument not specified
 */
static int
parse_resource_limit(const char *opt, enum submitType submit, unsigned int key,
                     char *arg, struct request *req)
{
    unsigned int i;

    for (i=0; i<req->number; i++) {

        if (req->keyvalues[i].key == key) {
            fprintf(stderr, "Cannot specify %s for multiple times", opt);
            return -1;
        }
    }

    unsigned int n = req->number;
    if (CMD_MODIFY_JOB == submit && 0 == strcmp(arg, "n")) {

        req->keyvalues[n].key = key;
        req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[n].value, 12, "%d", DELETE_NUMBER);
        req->number ++;

        return 0;
    }

    long limit = DEFAULT_RESOURCE_LIMIT;

    if (REQUEST_KEY_SUBMIT_CPULIMIT == key || REQUEST_KEY_SUBMIT_RUNLIMIT == key) {

        char *strHour = copy_string(arg);
        char *strMin = strchr(strHour, ':');
        if (NULL != strMin) {
            *strMin = '\0';
        }

        limit = gf_atoi(strHour, INFINIT_INT, -1);
        if (INFINIT_INT == limit) {

            fprintf(stderr, "%s has syntax error. Specified hour <%s> is not an integer",
                    opt, strHour);
            FREEUP(strHour);
            return -1;
        }

        if (NULL != strMin) {

            strMin ++;

            int minute;
            minute = gf_atoi(strMin, INFINIT_INT, -1);
            if (INFINIT_INT == minute) {

                fprintf(stderr, "%s has syntax error. Specified minute <%s> is not an integer",
                        opt, strMin);
                FREEUP(strHour);
                return -1;
            }

            limit *= 60;
            limit += minute;
        }
        limit *= 60; // make it second.
    } else {

        limit = gf_atoi(arg, INFINIT_INT, -1);
        if (INFINIT_INT == limit) {

            fprintf(stderr, "%s has syntax error. Specified value <%s> is not an integer",
                    opt, arg);
            return -1;
        }
    }

    req->keyvalues[n].key = key;
    req->keyvalues[n].value = (char *)calloc(24+1, sizeof(char));
    if (NULL == req->keyvalues[n].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", 24);
        return -1;
    }
    snprintf(req->keyvalues[n].value, 24, "%ld", limit);
    req->number ++;

    return 0;
} // end function prase_resource_limit

/**
 * @brief       parse ntask requirement and fill the submit request.
 *
 * @param[in]   arg           #1: ntask requirement string
 * @param[out]  req           #2: request to be filled
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: input option is wrong or kind of input argument not specified
 */
static int
parse_num_slots(const char *arg, struct request *req)
{
    char *strNumTask;
    strNumTask = copy_string(arg);

    int numThread=1;
    char *strThreadPerTask;
    strThreadPerTask = strchr(strNumTask, ':');
    if (NULL != strThreadPerTask) {
        *strThreadPerTask = '\0';
        strThreadPerTask ++;

        numThread = gf_atoi(strThreadPerTask, INFINIT_INT, 0);
        if (INFINIT_INT == numThread) {
            fprintf(stderr, "-n/--ntask argument has syntax error. The number of thread per task <%s> must be a positive integer",
                    strThreadPerTask);
            FREEUP(strNumTask);
            return -1;
        }
    }

    int numProcessor=1, maxProcessor=1;
    char *strMax;
    strMax = strchr(strNumTask, ',');
    if (NULL == strMax) {

        numProcessor = gf_atoi(strNumTask, INFINIT_INT, 0);
        if (INFINIT_INT == numProcessor) {
            fprintf(stderr, "-n/--ntask argument has syntax error. The number of processors <%s> must be a positive integer",
                    strNumTask);
            FREEUP(strNumTask);
            return -1;
        }

        maxProcessor = numProcessor;
    } else {

        *strMax = '\0';
        ++strMax;

        numProcessor = gf_atoi(strNumTask, INFINIT_INT, 0);
        if (INFINIT_INT == numProcessor) {
            fprintf(stderr, "-n/--ntask argument has syntax error. The number of processors <%s> must be a positive integer",
                    strNumTask);
            FREEUP(strNumTask);
            return -1;
        }

        maxProcessor = gf_atoi(strMax, INFINIT_INT, 0);
        if (INFINIT_INT == maxProcessor) {
            fprintf(stderr, "-n/--ntask argument has syntax error. The number of processors <%s> must be a positive integer",
                    strMax);
            FREEUP(strNumTask);
            return -1;
        }

        if (numProcessor > maxProcessor) {
            fprintf(stderr, "-n/--ntask argument has syntax error. Specified max task number %d is smaller than min task number %d", maxProcessor, numProcessor);
            FREEUP(strNumTask);
            return -1;
        }
    }

    FREEUP(strNumTask);

    req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_NUM_PROCESSOR;
    req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
    if (NULL == req->keyvalues[req->number].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
        return -1;
    }
    snprintf(req->keyvalues[req->number].value, 12, "%d", numProcessor);
    req->number ++;

    req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_MAX_PROCESSOR;
    req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
    if (NULL == req->keyvalues[req->number].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
        return -1;
    }
    snprintf(req->keyvalues[req->number].value, 12, "%d", maxProcessor);
    req->number ++;

    req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_THREAD_PERTASK;
    req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
    if (NULL == req->keyvalues[req->number].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
        return -1;
    }
    snprintf(req->keyvalues[req->number].value, 12, "%d", numThread);
    req->number ++;

    return 0;
} // end function parse_num_slots

/**
 * @brief       parse gpu requirement and fill it to the submit request.
 *
 * @param[in]   toparg        #1: gpu requirement string
 * @param[out]  req           #2: request to be filled
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: input option is wrong or kind of input argument not specified
 */
static int
parse_gpu(char *toptarg, struct request *req)
{
    bool_t numSpecified = FALSE;

    unsigned int len = strlen(toptarg);
    char *gpu = (char *)calloc(len+1, sizeof(char));
    if (NULL == gpu) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", len+1);
        return -1;
    }

    char *token;
    while (NULL != (token=get_string_token(&toptarg, ":", gpu, len+1))) {

        unsigned int tokenLen = strlen(token);
        char *key = (char *)calloc(tokenLen+1, sizeof(char));
        if (NULL == key) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", tokenLen+1);
            return -1;
        }

        char *tokenKey;
        tokenKey = get_string_token(&token, "=", key, tokenLen+1);
        if (NULL == tokenKey) {

            fprintf(stderr, "-O/--gpu argument has syntax error. Must specify a key");
            FREEUP(key);
            FREEUP(gpu);
            return -1;
        }

        unsigned int valueLen = strlen(token);
        char *value = (char *)calloc(valueLen+1, sizeof(char));
        if (NULL == value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", valueLen+1);
            return -1;
        }

        char *tokenValue;
        tokenValue = get_string_token(&token, "=", value, valueLen+1);
        if (NULL == tokenValue) {

            fprintf(stderr, "-O/--gpu requirement has syntax error. Must specify a value for key <%s>",
                    tokenKey);
            FREEUP(value);
            FREEUP(key);
            FREEUP(gpu);
            return -1;
        }

        if (strcmp(tokenKey, "num") == 0) {

            req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_GPU_NUM;
            req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
            if (NULL == req->keyvalues[req->number].value) {
                fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                return -1;
            }

            int numVal = gf_atoi(tokenValue, INFINIT_INT, 0);
            if (INFINIT_INT == numVal) {
                fprintf(stderr, "-O/--gpu requirement has syntax error. Must specify a positive value for key <%s>",
                        tokenKey);
                FREEUP(value);
                FREEUP(key);
                FREEUP(gpu);
                return -1;
            }

            snprintf(req->keyvalues[req->number].value, 12, "%d", numVal);
            numSpecified = TRUE;
            req->number ++;
        } else if (strcmp(tokenKey, "model") == 0) {

            req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_GPU_MODEL;
            req->keyvalues[req->number].value = copy_string(tokenValue);
            req->number ++;
        } else if (strcmp(tokenKey, "mps") == 0) {       

            if (!strcasecmp(tokenValue, "y") || !strcasecmp(tokenValue, "yes")
                || !strcasecmp(tokenValue, "n") || !strcasecmp(tokenValue, "no")) {

                req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_GPU_MPS;
                req->keyvalues[req->number].value = copy_string(tokenValue);
                req->number ++;
            } else {

                fprintf(stderr, "-O/--gpu requirement has syntax error. Must specify Yes|Y|No|N for key <mps>");
                FREEUP(value);
                FREEUP(key);
                FREEUP(gpu);
                return -1;
            }           
        } else if (strcmp(tokenKey, "mem") == 0) {

            req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_GPU_MEM;
            req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
            if (NULL == req->keyvalues[req->number].value) {
                fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                return -1;
            }

            float memVal = gf_atof(tokenValue, INFINIT_FLOAT, 0.0);
            if (INFINIT_FLOAT == memVal) {
                fprintf(stderr, "-O/--gpu requirement has syntax error. Must specify a positive value for key <%s>",
                        tokenKey);
                FREEUP(value);
                FREEUP(key);
                FREEUP(gpu);
                return -1;
            }

            snprintf(req->keyvalues[req->number].value, 12, "%.2f", atof(tokenValue));
            req->number ++;
        } else if (strcmp(tokenKey, "exclusive") == 0) {

            req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_GPU_EXCLUSIVE;
            if (!strcasecmp(tokenValue, "y") || !strcasecmp(tokenValue, "yes")) { 

                req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[req->number].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[req->number].value, 12, "%d", 1);
                req->number ++;   
            } else if (!strcasecmp(tokenValue, "n") || !strcasecmp(tokenValue, "no")) {

                req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[req->number].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[req->number].value, 12, "%d", 0);
                req->number ++;            
            } else {

                fprintf(stderr, "-O/--gpu requirement has syntax error. Must specify Yes|Y|No|N for key <exclusive>");
                FREEUP(value);
                FREEUP(key);
                FREEUP(gpu);
                return -1;
            }         
        } else {

            fprintf(stderr, "-O/--gpu requirement has syntax error. Specify an unknown key <%s>",
                    tokenKey);
            FREEUP(value);
            FREEUP(key);
            FREEUP(gpu);
            return -1;
        }

        FREEUP(value);
        FREEUP(key);
    }
    FREEUP(gpu);

    if (!numSpecified) { // num is not specified, set default 1.
        req->keyvalues[req->number].key = REQUEST_KEY_SUBMIT_GPU_NUM;
        req->keyvalues[req->number].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[req->number].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[req->number].value, 12, "%d", 1);
        req->number ++;
    }

    return 0;
} // end function parse_gpu

static void
job_sub_usage(enum submitType submit)
{
    switch(submit){
    case CMD_MODIFY_JOB:
        fprintf(stdout, "qp job mod usage: [-h,--help] [-B,--notify_begin | -Bn]\n");
        fprintf(stdout, "\t\t  [-N,--notify_end | -Nn] [-r,--rerunnable | -rn]\n");
        fprintf(stdout, "\t\t  [-x,--exclusive | -xn] [-H,--hold | -Hn]\n");
        fprintf(stdout, "\t\t  [-a,--extsub script_name]\n");
        fprintf(stdout, "\t\t  [-A,--postjob_command \"post_exec_command [argument ...]\" | -A,--postjob_command n]\n");
        fprintf(stdout, "\t\t  [-b,--begin_time [[month:]day:]hour:minute] | -b,--begin_time n]\n");
        fprintf(stdout, "\t\t  [-c,--cpulimit [hour:]minute] | -c,--cpulimit n]\n");
        fprintf(stdout, "\t\t  [-C,--corelimit core_limit | -C,--corelimit n]\n");
        fprintf(stdout, "\t\t  [-D,--datalimit data_limit | -D,--datalimit n]\n");
        fprintf(stdout, "\t\t  [-e,--error_file err_file | -e,--error_file n]\n");
        fprintf(stdout, "\t\t  [-E,--prejob_command \"pre_exec_command [argument ...]\" | -E,--prejob_command n]\n");
        fprintf(stdout, "\t\t  [-f,--file \"local_file op [remote_file]\" ... | -f,--file n]\n");
        fprintf(stdout, "\t\t  [-F,--filelimit file_limit | -F,--filelimit n]\n");
        fprintf(stdout, "\t\t  [-g,--jobgroup job_group_path | -g,--jobgroup n]\n");
        fprintf(stdout, "\t\t  [-G,--usergroup user_group | -G,--usergroup n]\n");
        fprintf(stdout, "\t\t  [-i,--input_file input_file | -i,--input_file n]\n");
        fprintf(stdout, "\t\t  [-j,--jobpriority priority | -j,--jobpriority n]\n");
        fprintf(stdout, "\t\t  [-J,--jobname job_name | -J,--jobname \"job_name[index_list]%%job_limit\" | -J,--jobname n]\n");
        fprintf(stdout, "\t\t  [-k,--chkpnt chkpnt_dir [period] [method=chkpnt_method] | -k,--chkpnt n]\n");
        fprintf(stdout, "\t\t  [-L,--loginshell login_shell | -L,--loginshell n]\n");
        fprintf(stdout, "\t\t  [-m,--node \"host_name[+[pref_level]] | host_group[+[pref_level]] ...\" | -m,--node n]\n");
        fprintf(stdout, "\t\t  [-M,--memlimit mem_limit | -M,--memlimit n]\n");
        fprintf(stdout, "\t\t  [-n,--ntask min[,max] | -n,--ntask n]\n");
        fprintf(stdout, "\t\t  [-o,--output_file out_file | -o,--output_file n]\n");
        fprintf(stdout, "\t\t  [-O,--gpu num=gpu_num[:model=gpu_model][:exclusive=y|n][:model=gpu_model][:mem=mem_value][:mps=y|n] | -O,--gpu n]\n");
        fprintf(stdout, "\t\t  [-P,--project project_name | -P,--project n]\n");
        fprintf(stdout, "\t\t  [-p,--processlimit process_limit | -p,--processlimit n]\n");
        fprintf(stdout, "\t\t  [-q,--queue \"queue_name ...\" | -q,--queue n]\n");
        fprintf(stdout, "\t\t  [-Q,--requeue_exitcode \"exit_code ...\" | -Q,--requeue_exitcode n]\n");
        fprintf(stdout, "\t\t  [-R,--requirement \"res_req\" | -R,--requirement n]\n");
        fprintf(stdout, "\t\t  [-s,--signal signal_value | -s,--signal n]\n");
        fprintf(stdout, "\t\t  [-S,--stacklimit stack_limit | -S,--stacklimit n]\n");
        fprintf(stdout, "\t\t  [-t,--term_time [[month:]day:]hour:minute | -t,--term_time n]\n");
        fprintf(stdout, "\t\t  [-u,--mail mail | -u,--mail n]\n");
        fprintf(stdout, "\t\t  [-v,--swaplimit swap_limit | -v,--swaplimit n]\n");
        fprintf(stdout, "\t\t  [-V,--cwd directory | -V,--cwd n]\n");
        fprintf(stdout, "\t\t  [-w,--dependency \"dependency_expression\" | -w,--dependency n]\n");
        fprintf(stdout, "\t\t  [-W,--runlimit run_limit | -W,--runlimit n]\n");
        fprintf(stdout, "\t\t  [-Z,--command \"new_command\" | -Z,--command n]\n");
        fprintf(stdout, "\t\t  [job_ID | \"job_ID[index]\"]\n");
        break;
    case CMD_RESTART_JOB:
        fprintf(stdout, "qp job restart usage: [-h,--help] [-B,--notify_begin]\n");
        fprintf(stdout, "\t\t      [-N,--notify_end] [-H,--hold] [-x,--exclusive]\n");
        fprintf(stdout, "\t\t      [-a,--extsub script_name]\n");
        fprintf(stdout, "\t\t      [-A,--postjob_command \"post_exec_command [argument ...]\"]\n");
        fprintf(stdout, "\t\t      [-b,--begin_time [[month:]day:]hour:minute]\n");
        fprintf(stdout, "\t\t      [-c,--cpulimit [hour:]minute]\n");
        fprintf(stdout, "\t\t      [-C,--corelimit core_limit]\n");
        fprintf(stdout, "\t\t      [-D,--datalimit data_limit]\n");
        fprintf(stdout, "\t\t      [-E,--prejob_command \"pre_exec_command [argument ...]\"]\n");
        fprintf(stdout, "\t\t      [-F,--filelimit file_limit] [-G,--usergroup user_group]\n");
        fprintf(stdout, "\t\t      [-m,--node \"host_name[+[pref_level]] | host_group[+[pref_level]] ...\"]\n");
        fprintf(stdout, "\t\t      [-M,--memlimit mem_limit] [-q,--queue \"queue_name ...\"]\n");
        fprintf(stdout, "\t\t      [-S,--stacklimit stack_limit]\n");
        fprintf(stdout, "\t\t      [-t,--term_time term_time]\n");
        fprintf(stdout, "\t\t      [-w,--dependency \"ependency_expression\"]\n");
        fprintf(stdout, "\t\t      [-v,--swaplimit swap_limit]\n");
        fprintf(stdout, "\t\t      [-V,--cwd directory]\n");
        fprintf(stdout, "\t\t      [-W,runlimit run_limit]\n");
        fprintf(stdout, "\t\t      chkpnt_dir [job_ID | \"job_ID[index]\"]\n");
        break;
    default:
        fprintf(stdout, "qp job sub usage: [-h,--help] [-B,--notify_begin]\n");
        fprintf(stdout, "\t\t  [-H,--hold] [-K,--block] [-N,--notify_end]\n");
        fprintf(stdout, "\t\t  [-r,--rerunnable] [-x,--exclusive]\n");
        fprintf(stdout, "\t\t  [-a,--extsub script_name]\n");
        fprintf(stdout, "\t\t  [-A,--postjob_command \"post_exec_command [argument ...]\"]\n");
        fprintf(stdout, "\t\t  [-b,--begin_time [[month:]day:]hour:minute]\n");
        fprintf(stdout, "\t\t  [-c,--cpulimit [hour:]minute]\n");
        fprintf(stdout, "\t\t  [-C,--corelimit core_limit]\n");
        fprintf(stdout, "\t\t  [-D,--datalimit data_limit]\n");
        fprintf(stdout, "\t\t  [-e,--error_file err_file]\n");
        fprintf(stdout, "\t\t  [-E,--prejob_command \"pre_exec_command [argument ...]\"]\n");
        fprintf(stdout, "\t\t  [-f,--file \"local_file op [remote_file]\" ...]\n");
        fprintf(stdout, "\t\t  [-F,--filelimit file_limit]\n");
        fprintf(stdout, "\t\t  [-g,--jobgroup job_group_path]\n");
        fprintf(stdout, "\t\t  [-G,--usergroup user_group]\n");
        fprintf(stdout, "\t\t  [-i,--input_file input_file]\n");
        fprintf(stdout, "\t\t  [-j,--jobpriority priority]\n");
        fprintf(stdout, "\t\t  [-J,--jobname job_name | -J,--jobname \"job_name[index_list]%%job_limit\"]\n");
        fprintf(stdout, "\t\t  [-k,--chkpnt chkpnt_dir [period] [method=chkpnt_method]]\n");
        fprintf(stdout, "\t\t  [-L,--loginshell login_shell]\n");
        fprintf(stdout, "\t\t  [-m,--node \"host_name[+[pref_level]] | host_group[+[pref_level]] ...\"]\n");
        fprintf(stdout, "\t\t  [-M,--memlimit mem_limit] [-n,--ntask min[,max]]\n");
        fprintf(stdout, "\t\t  [-o,--output_file out_file]\n");
        fprintf(stdout, "\t\t  [-O,--gpu num=gpu_num[:model=gpu_model][:exclusive=y|n][:mem=mem_value][:mps=y|n]]\n");
        fprintf(stdout, "\t\t  [-p,--processlimit process_limit]\n");
        fprintf(stdout, "\t\t  [-P,--project project_name]\n");
        fprintf(stdout, "\t\t  [-q,--queue \"queue_name ...\"]\n");
        fprintf(stdout, "\t\t  [-Q,--requeue_exitcode \"exit_code ...\"]\n");
        fprintf(stdout, "\t\t  [-R,--requirement \"res_req\"]\n");
        fprintf(stdout, "\t\t  [-s,--signal signal_value]\n");
        fprintf(stdout, "\t\t  [-S,--stacklimit stack_limit]\n");
        fprintf(stdout, "\t\t  [-t,--term_time [[month:]day:]hour:minute]\n");
        fprintf(stdout, "\t\t  [-u,--mail mail_user]\n");
        fprintf(stdout, "\t\t  [-v,--swaplimit swap_limit | -v,--swaplimit n]\n");
        fprintf(stdout, "\t\t  [-V,--cwd directory]\n");
        fprintf(stdout, "\t\t  [-w,--dependency \"dependency_expression\"]\n");
        fprintf(stdout, "\t\t  [-W,--runlimit [hours:]minutes]\n");
        break;
    }

    exit(0);
} // end function job_sub_usage

/**
 * @brief       parse every submit option and fill the submit request.
 *
 * @param[in]   argc          #1: number of input argument
 * @param[in]   argv          #2: string array of the input argument
 * @param[in]   submit        #3: submit, modify or restart a job
 * @param[out]  req           #4: request to be filled
 *
 * @retval      0             #1: succeed
 * @retval      -1            #2: input option is wrong or kind of input argument not specified
 */
static int
fill_submit_request(int argc, char **argv, enum submitType submit, struct request *req)
{
    unsigned int deleteOptions=0, deleteOptions2=0;

    if (CMD_MODIFY_JOB == submit) {
        int i, j;
        bool_t foundN=FALSE;

        for (i=0; i<argc; i++) {

            if (strcmp(argv[i], "-Bn") == 0) {
                deleteOptions |= SUBOPT_NOTIFY_BEGIN;
                foundN = TRUE;
            } else if (strcmp(argv[i], "-Hn") == 0) {
                deleteOptions2 |= SUBOPT2_HOLD;
                foundN = TRUE;
            } else if (strcmp(argv[i], "-Nn") == 0) {
                deleteOptions |= SUBOPT_NOTIFY_END;
                foundN = TRUE;
            } else if (strcmp(argv[i], "-rn") == 0) {
                deleteOptions |= SUBOPT_RERUNNABLE;
                foundN = TRUE;
            } else if (strcmp(argv[i], "-xn") == 0) {
                deleteOptions |= SUBOPT_EXCLUSIVE;
                foundN = TRUE;
            }

            if (foundN) {
                for (j=i;j<argc-1; j++) {
                    argv[j] = argv[j+1];
                }

                i --;
                argc --;
                foundN = FALSE;

                continue;
            }
        }
    }

    const char *options;
    static struct option long_options[200];
    unsigned int num=0;
    long_options[num].name = "extsub";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'a';
    num ++;

    long_options[num].name = "begin_time";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'b';
    num ++;

    long_options[num].name = "notify_begin";
    long_options[num].has_arg = no_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'B';
    num ++;

    long_options[num].name = "cpulimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'c';
    num ++;

    long_options[num].name = "corelimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'C';
    num ++;

    long_options[num].name = "datalimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'D';
    num ++;

    long_options[num].name = "prejob_command";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'E';
    num ++;

    long_options[num].name = "postjob_command";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'A';
    num ++;

    long_options[num].name = "usergroup";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'G';
    num ++;

    long_options[num].name = "file";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'f';
    num ++;

    long_options[num].name = "filelimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'F';
    num ++;

    long_options[num].name = "help";
    long_options[num].has_arg = no_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'h';
    num ++;

    long_options[num].name = "hold";
    long_options[num].has_arg = no_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'H';
    num ++;

    long_options[num].name = "cwd";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'V';
    num ++;

    long_options[num].name = "node";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'm';
    num ++;

    long_options[num].name = "memlimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'M';
    num ++;

    long_options[num].name = "queue";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'q';
    num ++;

    long_options[num].name = "notify_end";
    long_options[num].has_arg = no_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'N';
    num ++;

    long_options[num].name = "dependency";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'w';
    num ++;

    long_options[num].name = "runlimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'W';
    num ++;

    long_options[num].name = "stacklimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'S';
    num ++;

    long_options[num].name = "term_time";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 't';
    num ++;

    long_options[num].name = "exclusive";
    long_options[num].has_arg = no_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'x';
    num ++;

    long_options[num].name = "swaplimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'v';
    num ++;

    long_options[num].name = "processlimit";
    long_options[num].has_arg = required_argument;
    long_options[num].flag = 0;
    long_options[num].val = 'p';
    num ++;

    if (CMD_RESTART_JOB == submit) {

        options = "a:A:E:b:c:f:w:m:q:t:V:W:F:D:S:p:v:C:M:G:BxNhH";

        long_options[num].name = NULL;
        long_options[num].has_arg = no_argument;
        long_options[num].flag = 0;
        long_options[num].val = 0;
    } else {

        if (CMD_MODIFY_JOB == submit) {
            options = "a:A:E:b:c:f:w:m:q:Q:t:V:W:F:D:S:p:v:C:M:G:e:g:i:O:J:k:L:n:o:j:s:R:u:P:Z:BxNhHr";
        } else {
            options = "a:A:E:b:c:f:w:m:q:Q:t:V:W:F:D:S:p:v:C:M:G:e:g:i:O:J:k:L:n:o:j:s:R:u:P:BxNhHKr";
        }

        long_options[num].name = "error_file";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'e';
        num ++;

        long_options[num].name = "jobgroup";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'g';
        num ++;

        long_options[num].name = "input_file";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'i';
        num ++;

        long_options[num].name = "gpu";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'O';
        num ++;

        long_options[num].name = "jobname";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'J';
        num ++;

        long_options[num].name = "chkpnt";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'k';
        num ++;

        if (CMD_SUBMIT_JOB == submit) {
            long_options[num].name = "block";
            long_options[num].has_arg = required_argument;
            long_options[num].flag = 0;
            long_options[num].val = 'K';
            num ++;
        }

        long_options[num].name = "loginshell";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'L';
        num ++;

        long_options[num].name = "ntask";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'n';
        num ++;

        long_options[num].name = "output_file";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'o';
        num ++;

        long_options[num].name = "jobpriority";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'j';
        num ++;

        long_options[num].name = "signal";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 's';
        num ++;

        long_options[num].name = "requirement";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'R';
        num ++;

        long_options[num].name = "mail";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'u';
        num ++;

        long_options[num].name = "project";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'P';
        num ++;

        long_options[num].name = "rerunnable";
        long_options[num].has_arg = no_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'r';
        num ++;

        long_options[num].name = "requeue_exitcode";
        long_options[num].has_arg = required_argument;
        long_options[num].flag = 0;
        long_options[num].val = 'Q';
        num ++;

        if (CMD_MODIFY_JOB == submit) {

            long_options[num].name = "command";
            long_options[num].has_arg = required_argument;
            long_options[num].flag = 0;
            long_options[num].val = 'Z';
            num ++;
        }

        long_options[num].name = NULL;
        long_options[num].has_arg = no_argument;
        long_options[num].flag = 0;
        long_options[num].val = 0;
    }

    unsigned int n=req->number, submitOptions=0, submitOptions2=0;
    req->keyvalues[n].key = REQUEST_KEY_SUBMIT_TYPE;
    req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
    if (NULL == req->keyvalues[n].value) {
        fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
        return -1;
    }
    snprintf(req->keyvalues[n].value, 12, "%d", submit);
    n++;

    if (submit == CMD_RESTART_JOB) {
        submitOptions |= SUBOPT_RESTART;
    }

    unsigned int i;
    bool_t arrayJob = FALSE;
    while (TRUE) {

        /* TODO: control this behavior by parameter if needed.
         * If argv[optind] is not prefixed with '-', it is start of job command.
         * stop parsing options.
         */
        if (optind < argc && argv[optind][0] != '-') {
            break;
        }

        int option_index = 0;
        int opt;
        opt = getopt_long(argc, argv, options, long_options, &option_index);
        if (-1 == opt) {
            break;
        }

        switch (opt) {
        case 'a':

            if (0 == strcmp(optarg, "none")) {
                submitOptions |= SUBOPT_NOFORK;
            }

            break;
        case 'A':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_POSTCMD) {
                    fprintf(stderr, "Cannot specify -A/--postjob_command for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_POST_EXEC;
                break;
            }
                
            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_POSTCMD;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'b':
        {
            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_BEGINTIME) {
                    fprintf(stderr, "Cannot specify -b/--begin_time for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {

                req->keyvalues[n].key = REQUEST_KEY_SUBMIT_BEGINTIME;
                req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[n].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[n].value, 12, "%d", DELETE_NUMBER);
                n++;
                break;
            }

            time_t beginTime = 0;
            beginTime = optarg_2time(optarg);
            if (0 > beginTime) {
                return -1;
            }

            time_t now = time(NULL);
            if (now > beginTime) {
                log_message(ERR, NOCLASS, "%s: cannot specified a time <%s> ahead of now",
                            __func__, optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_BEGINTIME;
            req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
            if (NULL == req->keyvalues[n].value) {
                fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                return -1;
            }
            snprintf(req->keyvalues[n].value, 12, "%ld", beginTime);
            n++;

            break;
        }
        case 'B':

            if (submit == CMD_MODIFY_JOB && (deleteOptions & SUBOPT_NOTIFY_BEGIN)) {
                fprintf(stderr, "Cannot specify -Bn together with -B/--notify_begin");
                return -1;
            }

            submitOptions |= SUBOPT_NOTIFY_BEGIN;

            break;
        case 'c':

            req->number = n;
            if (parse_resource_limit("c/--cpulimit", submit, REQUEST_KEY_SUBMIT_CPULIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'C':

            req->number = n;
            if (parse_resource_limit("C/--corelimit", submit, REQUEST_KEY_SUBMIT_CORELIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'D':

            req->number = n;
            if (parse_resource_limit("D/--datalimit", submit, REQUEST_KEY_SUBMIT_DATALIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'e':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_ERRFILE) {
                    fprintf(stderr, "Cannot specify -e/--error_file for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_ERR_FILE;
                break;
            }

            if (strlen(optarg) > PATH_MAX - 1) {
                fprintf(stderr, "%s: file name is too long", optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_ERRFILE;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'E':

            for (i=0; i<n; i++) {
                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_PRECMD) {
                    fprintf(stderr, "Cannot specify -E/--prejob_command for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_PRE_EXEC;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_PRECMD;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'f':

            for (i=0; i<n; i++) {
                if (req->keyvalues[i].key != REQUEST_KEY_SUBMIT_NUM_XFILE) {
                    continue;
                }

                if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                    fprintf(stderr, "Cannot specify both -f/--file n and -f/--file");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_OTHER_FILES;
                break;
            }

            req->number = n;
            if (parse_execution_file(optarg, req) < 0) {
                return -1;
            }
            n = req->number;
            submitOptions |= SUBOPT_OTHER_FILES;

            break;
        case 'F':

            req->number = n;
            if (parse_resource_limit("F/--filelimit", submit, REQUEST_KEY_SUBMIT_FILELIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'g':

            for (i=0; i<n; i++) {
                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_JOBGROUP) {
                    fprintf(stderr, "Cannot specify -g/--jobgroup for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_JOB_GROUP;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_JOBGROUP;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

	    break;
        case 'G':

            for (i=0; i<n; i++) {
                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_USERGROUP) {
                    fprintf(stderr, "Cannot specify -G/--usergroup for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_USER_GROUP;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_USERGROUP;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'h':

            job_sub_usage(submit);

	    return -1;
        case 'H':

            if (submit == CMD_MODIFY_JOB && (deleteOptions2 & SUBOPT2_HOLD)) {
                fprintf(stderr, "Cannot specify -Hn together with -H/--hold");
                return -1;
            }

            submitOptions2 |= SUBOPT2_HOLD;

            break;
        case 'i':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_INFILE) {
                    fprintf(stderr, "Cannot specify -i/--input_file for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_IN_FILE;
                break;
            }

            if (strlen(optarg) > PATH_MAX - 1) {
                fprintf(stderr, "%s: file name is too long", optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_INFILE;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'j':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_JOBPRIORITY) {
                    fprintf(stderr, "Cannot specify -j/--jobpriority for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_JOB_PRIORITY;
                break;
            }

            if (!isinteger(optarg) || atoi(optarg) <= 0) {
                fprintf(stderr, "%s: job priority is not an integer or a positive integer", optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_JOBPRIORITY;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'J':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_JOBNAME) {
                    fprintf(stderr, "Cannot specify -J/--jobname for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_JOB_NAME;
                break;
            }

            if (NULL != strchr(optarg, '[')) {
                arrayJob = TRUE;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_JOBNAME;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'k':
        {
            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_CHKPNT_DIR) {
                    fprintf(stderr, "Cannot specify -k/--chkpnt for multiple times");
                    return -1;
                }
            }
            
            char *sp = optarg;
            unsigned int len = strlen(optarg);
            char *strChkpnt = (char *)calloc(len+1, sizeof(char));
            if (NULL == strChkpnt) {
                fprintf(stderr, "calloc(%d) failed due to %m.\n", len+1);
                return -1;
            }

            char *token;
            token = get_string_token(&sp, " ", strChkpnt, len+1);
            if (NULL == token) {
                fprintf(stderr, "Must specify valid chkpnt directory");
                FREEUP(strChkpnt);
                return -1;
            }
            len = strlen(sp);

            req->keyvalues[n].key = REQUEST_KEY_CHKPNT_DIR;
            req->keyvalues[n].value = copy_string(token);
            n++;

            token = get_string_token(&sp, " ", strChkpnt, len+1);
            if (NULL == token) {
                submitOptions |= SUBOPT_CHKPNT_DIR;
                FREEUP(strChkpnt);
                break;
            }
            len = strlen(sp);

            unsigned int chkperiod = 0;
            if (isinteger(token)) {

                chkperiod = gf_atoi(token, INFINIT_INT, -1);
                if (INFINIT_INT == chkperiod) {
                    fprintf(stderr, "Bad checkpoint period value <%s>, valid value is between 0-%d",
                            token, INFINIT_INT-1);
                    FREEUP(strChkpnt);
                    return -1;
                }

                chkperiod *= 60; // turn it to second
                submitOptions |= SUBOPT_CHKPNT_PERIOD;

                req->keyvalues[n].key = REQUEST_KEY_CHKPNT_PERIOD;
                req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[n].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[n].value, 12, "%d", chkperiod);
                n++;

                token = get_string_token(&sp, " ", strChkpnt, len+1);
                if (NULL == token) {
                    FREEUP(strChkpnt);
                    break;
                }
                len = strlen(sp);

                if (strstr(token, "method=") == token) {
                    char *strMethod;

                    strMethod = strchr(token, '=');
                    if (*(++strMethod) != '\0') {
                        req->keyvalues[n].key = REQUEST_KEY_CHKPNT_METHOD;
                        req->keyvalues[n].value = copy_string(strMethod);
                        n++;
                    } else {
                        fprintf(stderr, "%s: -k/--chkpnt argrument has syntax error. Correct syntax is qp job sub -k/--chkpnt \"chkpnt_dir [period] [method=checkpoint method]\"", token);
                        FREEUP(strChkpnt);
                        return -1;
                    }

                    submitOptions |= SUBOPT_CHKPNT_PERIOD;
                } else {
                    fprintf(stderr, "%s: -k/--chkpnt argument has yntax error. Correct syntax is qp job sub -k/--chkpnt \"chkpnt_dir [period] [method=checkpoint method]\"", token);
                    FREEUP(strChkpnt);
                    return -1;
                }
            } else if (strstr(token, "method=") == token) {
                char *strMethod;

                strMethod = strchr(token, '=');
                if (*(++strMethod) != '\0') {
                    req->keyvalues[n].key = REQUEST_KEY_CHKPNT_METHOD;
                    req->keyvalues[n].value = copy_string(strMethod);
                    n++;
                } else {
                    fprintf(stderr, "%s: -k/--chkpnt argrument has syntax error. Correct syntax is qp job sub -k/--chkpnt \"chkpnt_dir [period] [method=checkpoint method]\"", token);
                    FREEUP(strChkpnt);
                    return -1;
                }

                submitOptions |= SUBOPT_CHKPNT_PERIOD;
            } else {
                fprintf(stderr, "%s: -k/--chkpnt argrument has syntax error. Correct syntax is qp job sub -k/--chkpnt \"chkpnt_dir [period] [method=checkpoint method]\"", token);
                FREEUP(strChkpnt);
                return -1;
            }

            token = get_string_token(&sp, " ", strChkpnt, len+1);
            if (NULL != token) {
                fprintf(stderr, "%s: specify too many argument for -k/--chkpnt. Correct syntax is qp job sub -k/--chkpnt \"chkpnt_dir [period] [method=checkpoint method]\"", token);
                FREEUP(strChkpnt);
                return -1;
            }

            FREEUP(strChkpnt);

            break;           
        }
        case 'K':

            submitOptions2 |= SUBOPT2_BLOCK;

            break;
        case 'L':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_LOGINSHELL) {
                    fprintf(stderr, "Cannot specify -L/--loginshell for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_LOGIN_SHELL;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_LOGINSHELL;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'm':
        {
            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_NODE) {
                    fprintf(stderr, "Cannot specify -m/--node for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_HOST;
                break;
            }

            char *sp = optarg;
            strip_spaces(sp, FALSE);

            if (strlen(sp) <= 0) {
                fprintf(stderr, "No valid host specified in -m/--node option");
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_NODE;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        }
        case 'M':

            req->number = n;
            if (parse_resource_limit("M/--memlimit", submit, REQUEST_KEY_SUBMIT_RSSLIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'n':
        {
            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_NUM_PROCESSOR) {
                    fprintf(stderr, "Cannot specify -n/--ntask for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {

                req->keyvalues[n].key = REQUEST_KEY_SUBMIT_NUM_PROCESSOR;
                req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[n].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[n].value, 12, "%d", DEL_NUMPRO);
                n++;

                req->keyvalues[n].key = REQUEST_KEY_SUBMIT_MAX_PROCESSOR;
                req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[n].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[n].value, 12, "%d", DEL_NUMPRO);
                n++;
                break;
            }

            req->number = n;
            if (parse_num_slots(optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        }
        case 'N':

            if (submit == CMD_MODIFY_JOB && (deleteOptions & SUBOPT_NOTIFY_END)) {
                fprintf(stderr, "Cannot specify -Nn together with -N/--notify_end");
                return -1;
            }

            submitOptions |= SUBOPT_NOTIFY_END;

            break;
        case 'o':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_OUTFILE) {
                    fprintf(stderr, "Cannot specify -o/--output_file for multiple times");
                    return -1;
                }
            }

            if (strlen(optarg) > PATH_MAX - 1) {
                fprintf(stderr, "%s: file name is too long.\n", optarg);
                return -1;
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_OUT_FILE;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_OUTFILE;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'O':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_GPU_NUM) {
                    fprintf(stderr, "Cannot specify -O/--gpu for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_GPU;
                break;
            }

            req->number = n;
            if (parse_gpu(optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'p':

            req->number = n;
            if (parse_resource_limit("p/--processlimit", submit, REQUEST_KEY_SUBMIT_PROCESSLIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'P':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_PROJECT) {
                    fprintf(stderr, "Cannot specify -P/--project for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_PROJECT_NAME;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_PROJECT;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'q':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_QUEUE) {
                    fprintf(stderr, "Cannot specify -q/--queue for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_QUEUE;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_QUEUE;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'Q':

            for (i = 0; i < n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_REQUEUE_EXITCODE) {
                    fprintf(stderr, "Cannot specify -Q/--requeue_exitcode for multiple times");
                    return -1;
                }
            }
            
            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions2 |= SUBOPT2_REQUEUE_EXITCODE;
                break;
            }
            
            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_REQUEUE_EXITCODE;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'r':

            if (submit == CMD_MODIFY_JOB && (deleteOptions & SUBOPT_RERUNNABLE)) {
                fprintf(stderr, "Cannot specify -rn together with -r/--rerunnable");
                return -1;
            }

            submitOptions |= SUBOPT_RERUNNABLE;

            break;
        case 'R':

            for (i=0; i<n; i++) {
                if (req->keyvalues[i].key != REQUEST_KEY_SUBMIT_RESREQ) {
                    continue;
                }

                if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                    fprintf(stderr, "Cannot specify both -R/--requirement n and -R/--requirement");
                    return -1;
                }
            }
            
            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_RES_REQ;
                break;
            }

            while (*optarg == ' ')
                optarg++;

            if (i < n) {
                if (0 > merge_requirement_string(&(req->keyvalues[i].value), optarg)) {
                    fprintf(stderr, "Resource requirement is too long or invalid");
                    return -1;
                }
            } else {
                req->keyvalues[n].key = REQUEST_KEY_SUBMIT_RESREQ;
                req->keyvalues[n].value = copy_string(optarg);
                n++;
            }

            break;
        case 's':
        {
            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_SIGNAL) {
                    fprintf(stderr, "Cannot specify -s/--signal for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_WINDOW_SIG;
                break;
            }

            int signalValue = gf_string_2signal(optarg);
            if (signalValue < 0) {
                fprintf(stderr, "%s: Illegal signal value", optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_SIGNAL;
            req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
            if (NULL == req->keyvalues[n].value) {
                fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                return -1;
            }
            snprintf(req->keyvalues[n].value, 12, "%d", signalValue);
            n++;

            break;
        }
        case 'S':

            req->number = n;
            if (parse_resource_limit("S/--stacklimit", submit, REQUEST_KEY_SUBMIT_STACKLIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 't':
        {
            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_TERMTIME) {
                    fprintf(stderr, "Cannot specify -t/--term_time for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {

                req->keyvalues[n].key = REQUEST_KEY_SUBMIT_TERMTIME;
                req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
                if (NULL == req->keyvalues[n].value) {
                    fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                    return -1;
                }
                snprintf(req->keyvalues[n].value, 12, "%d", DELETE_NUMBER);
                n++;
                break;
            }

            time_t termTime;
            termTime = optarg_2time(optarg);
            if (0 > termTime) {
                return -1;
            }

            time_t now = time(NULL);
            if (now > termTime) {
                log_message(ERR, NOCLASS, "%s: cannot specified a time <%s> ahead of now",
                            __func__, optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_TERMTIME;
            req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
            if (NULL == req->keyvalues[n].value) {
                fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
                return -1;
            }
            snprintf(req->keyvalues[n].value, 12, "%ld", termTime);
            n++;

            break;
        }
        case 'u':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_MAIL) {
                    fprintf(stderr, "Cannot specify -u/--mail for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_MAIL_USER;
                break;
            }

            if (strlen(optarg) > MAXHOSTNAMELEN - 1) {
                fprintf(stderr, "%s: Mail destination name is too long", optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_MAIL;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'v':

            req->number = n;
            if (parse_resource_limit("v/--swaplimit", submit, REQUEST_KEY_SUBMIT_SWAPLIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'V':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_CWD) {
                    fprintf(stderr, "Cannot specify -V/--cwd for multiple times.\n");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions2 |= SUBOPT2_CWD;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_CWD;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'w':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_SUBMIT_DEPENDENCY) {
                    fprintf(stderr, "Cannot specify -w/--dependency for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions |= SUBOPT_DEPEND_COND;
                break;
            }

            req->keyvalues[n].key = REQUEST_KEY_SUBMIT_DEPENDENCY;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        case 'W':

            req->number = n;
            if (parse_resource_limit("W/--runlimit", submit, REQUEST_KEY_SUBMIT_RUNLIMIT,
                                     optarg, req) < 0) {
                return -1;
            }
            n = req->number;

            break;
        case 'x':

            if (submit == CMD_MODIFY_JOB && (deleteOptions & SUBOPT_EXCLUSIVE)) {
                fprintf(stderr, "Cannot specify -xn together with -x/--exclusive");
                return -1;
            }

            submitOptions |= SUBOPT_EXCLUSIVE;

            break;
        case 'Z':

            for (i=0; i<n; i++) {

                if (req->keyvalues[i].key == REQUEST_KEY_MODIFY_COMMAND) {
                    fprintf(stderr, "Cannot specify -Z/--command for multiple times");
                    return -1;
                }
            }

            if (submit == CMD_MODIFY_JOB && strcmp(optarg, "n") == 0) {
                deleteOptions2 |= SUBOPT2_MODIFY_CMD;
                break;
            }

            if (strlen(optarg) >= PATH_MAX) {
                fprintf(stderr, "%s: command is too long", optarg);
                return -1;
            }

            req->keyvalues[n].key = REQUEST_KEY_MODIFY_COMMAND;
            req->keyvalues[n].value = copy_string(optarg);
            n++;

            break;
        default:

            job_sub_usage(submit);
            return -1;
        }
    }

    if (submitOptions2 & SUBOPT2_BLOCK && arrayJob) {
        fprintf(stderr, "Job array doesn't support -K/--block option");
        return -1;
    }

    if (0 < submitOptions) {
        req->keyvalues[n].key = REQUEST_KEY_SUBMIT_OPTIONS;
        req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[n].value, 12, "%d", submitOptions);
        n++;
    }

    if (0 < submitOptions2) {
        req->keyvalues[n].key = REQUEST_KEY_SUBMIT_OPTIONS2;
        req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[n].value, 12, "%d", submitOptions2);
        n++;
    }

    if (0 < deleteOptions) {
        req->keyvalues[n].key = REQUEST_KEY_MODIFY_DELETE_OPTIONS;
        req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[n].value, 12, "%d", deleteOptions);
        n++;
    }

    if (0 < deleteOptions2) {
        req->keyvalues[n].key = REQUEST_KEY_MODIFY_DELETE_OPTIONS2;
        req->keyvalues[n].value = (char *)calloc(12, sizeof(char));
        if (NULL == req->keyvalues[n].value) {
            fprintf(stderr, "calloc(%d) failed due to %m.\n", 12);
            return -1;
        }
        snprintf(req->keyvalues[n].value, 12, "%d", deleteOptions2);
        n++;
    }

    req->number = n;

    return 0;
} // end function fill_submit_request
